{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Success/Errors Histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import numpy as np\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, '../')\n",
    "from loaders import get_loader\n",
    "from learners import get_learner\n",
    "from utils import misc\n",
    "\n",
    "# Device configuration\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DEFINE HERE YOUR CONFIG PATH AND THE EPOCH YOU WANT TO EVALUATE\n",
    "config_path = '/data/DEEPLEARNING/ccorbiere/logs/cifar10/github_weights/baseline/config_1.yaml' \n",
    "ckpt_path = '/data/DEEPLEARNING/ccorbiere/logs/cifar10/github_weights/baseline/model_epoch_197.ckpt'\n",
    "config_args = misc.load_yaml(config_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-22 21:20:40,127 gpuserver3 confidnet.loaders.loader[17796] INFO --- Augmentations ---\n",
      "2019-12-22 21:20:40,129 gpuserver3 confidnet.augmentations[17796] INFO Using hflip aug with params True\n",
      "2019-12-22 21:20:40,130 gpuserver3 confidnet.augmentations[17796] INFO Using rotate aug with params 15\n",
      "2019-12-22 21:20:40,131 gpuserver3 confidnet.augmentations[17796] INFO Using No Augmentations\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-22 21:20:42,056 gpuserver3 confidnet.loaders.loader[17796] WARNING Loading existing train-val split indices\n"
     ]
    }
   ],
   "source": [
    "# Get data loader\n",
    "dloader = get_loader(config_args)\n",
    "dloader.make_loaders()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2019-12-22 21:20:51,473 gpuserver3 confidnet.learners.learner[17796] INFO Using optimizer adam\n",
      "2019-12-22 21:20:51,475 gpuserver3 confidnet.learners.learner[17796] INFO Using loss cross_entropy\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Initialize and load model\n",
    "learner = get_learner(config_args, dloader.train_loader, dloader.val_loader, dloader.test_loader, -1, device)\n",
    "checkpoint = torch.load(ckpt_path)\n",
    "learner.model.load_state_dict(checkpoint[\"model_state_dict\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_test_set(learner, mode='normal', samples=50, verbose=True):\n",
    "    accurate, errors, confidence = [], [], []\n",
    "    loop = tqdm(learner.test_loader, disable=not verbose)\n",
    "    for batch_id, (data, target) in enumerate(loop):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            if mode == \"normal\":\n",
    "                output = learner.model(data)\n",
    "                conf, pred = F.softmax(output, dim=1).max(dim=1, keepdim=True)\n",
    "\n",
    "            elif mode == \"gt\":\n",
    "                output = learner.model(data)\n",
    "                probs = F.softmax(output, dim=1)\n",
    "                pred = probs.max(dim=1, keepdim=True)[1]\n",
    "                labels_hot = misc.one_hot_embedding(target, learner.num_classes).to(device)\n",
    "                # Segmentation special case\n",
    "                if learner.task == \"segmentation\":\n",
    "                    labels_hot = labels_hot.permute(0, 3, 1, 2)\n",
    "                conf, _ = (labels_hot * probs).max(dim=1, keepdim=True)\n",
    "\n",
    "            elif mode == \"mc_dropout\":\n",
    "                if learner.task == \"classification\":\n",
    "                    outputs = torch.zeros(samples, data.shape[0], learner.num_classes).to(device)\n",
    "                elif learner.task == \"segmentation\":\n",
    "                    outputs = torch.zeros(samples, data.shape[0], learner.num_classes, \n",
    "                                          data.shape[2], data.shape[3]).to(device)\n",
    "                for i in range(samples):\n",
    "                    outputs[i] = learner.model(data)\n",
    "                output = outputs.mean(0)\n",
    "                probs = F.softmax(output, dim=1)\n",
    "                conf = (probs * torch.log(probs + 1e-9)).sum(dim=1)  # entropy\n",
    "                pred = probs.max(dim=1, keepdim=True)[1]\n",
    "\n",
    "            elif mode=='confidnet':\n",
    "                output, conf = learner.model(data)\n",
    "                pred = output.argmax(dim=1, keepdim=True)\n",
    "                conf = torch.sigmoid(conf)\n",
    "\n",
    "            accurate.extend(pred.eq(target.view_as(pred)))\n",
    "            errors.extend(pred!=target.view_as(pred))\n",
    "            confidence.extend(conf) \n",
    "\n",
    "    confidence = torch.stack(confidence).detach().to('cpu').numpy() .flatten()  \n",
    "    accurate = torch.cat(accurate).detach().to('cpu').numpy().flatten()\n",
    "    errors = torch.cat(errors).detach().to('cpu').numpy().flatten()\n",
    "    ap_success = average_precision_score(accurate, confidence)\n",
    "    ap_errors = average_precision_score(errors, -confidence)\n",
    "\n",
    "    return accurate, confidence, ap_success, ap_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f20bdf3c27de4602a1f16a0b9bd4cf97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=79), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accurate_normal, confidence_normal, _, _ = predict_test_set(learner, mode='normal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7d7bfce0621498a883ce68faa605ef5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=79), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "accurate_gt, confidence_gt, _, _ = predict_test_set(learner, mode='gt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4YAAAFRCAYAAAAo17OzAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZgcZbX48e8hYZFFURIUjBhQQJaEAJFdCcq+CqLAD5BNuS5sehXBDRTxcl3uVcRrjBBBZXMBiYBsiiAYlIQl7IuAEoMSloRdSDi/P6oGO0PPTPeke7qn+/t5nnmmq+rtqtNvkj45b1W9FZmJJEmSJKl7LdHqACRJkiRJrWVhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlRrY6gEYaNWpUjh07ttVhSJKabObMmY9l5uhWxzFcmB8lqXsMNkd2VGE4duxYZsyY0eowJElNFhF/bXUMw4n5UZK6x2BzpJeSSpIkSVKXszCUJEmSpC5nYShJkiRJXa6j7jGs5qWXXmL27Nm88MILrQ6loyyzzDKMGTOGJZdcstWhSJIGwfzYPOZIScNRxxeGs2fPZoUVVmDs2LFERKvD6QiZyeOPP87s2bNZffXVWx2OJGkQzI/NYY6UNFx1/KWkL7zwAiuttJJJr4EigpVWWslRZkkaxsyPzWGOlDRcdXxhCJj0msA+laThz+/y5rBfJQ1HXVEYttrJJ5/Meuutx/jx45kwYQJ/+tOfWh2SJEltwRwpSe2h4+8x7C2+3NhRvDwh+90+ffp0Lr74Ym666SaWXnppHnvsMV588cWGxiBJ0uIa6vwI5khJaieeMWyyRx55hFGjRrH00ksDMGrUKFZddVXGjh3LY489BsCMGTOYNGkSAM888wyHHHII48aNY/z48fzyl78E4LLLLmOjjTZigw024L3vfS8Azz77LIceeijvfOc72XDDDbnooosAuOOOO9hkk02YMGEC48eP57777uPZZ59ll112YYMNNmD99dfn/PPPB2DmzJlsvfXWbLzxxuywww488sgjAJx66qmsu+66jB8/nn333XfI+kuS1D3MkZLUPrrujOFQ23777fnKV77CWmutxbbbbss+++zD1ltv3Wf7k046ide97nXcdtttADz55JPMnTuXj3zkI1x77bWsvvrqPPHEE0Bx+c173vMepk6dyrx589hkk03YdtttmTx5MkcffTT7778/L774IgsXLuTSSy9l1VVX5ZJLLgFg/vz5vPTSSxx55JFcdNFFjB49mvPPP5/Pf/7zTJ06lVNOOYUHH3yQpZdemnnz5jW/oyRJXcccKUntw8KwyZZffnlmzpzJH/7wB66++mr22WcfTjnllD7bX3XVVZx33nmvLL/+9a/n17/+Ne9+97tfmfb6DW94AwBXXHEF06ZN45vf/CZQzDD3t7/9jc0335yTTz6Z2bNns9dee7Hmmmsybtw4Pv3pT/PZz36WXXfdlXe9613cfvvt3H777Wy33XYALFy4kFVWWQWA8ePHs//++/O+972P973vfU3pG0ndpdGXKmr4M0dKUqEdcqSF4RAYMWIEkyZNYtKkSYwbN46zzjqLkSNH8vLLLwMsMqV1Zr5qNrNq63rW//KXv2TttddeZP0666zDpptuyiWXXMIOO+zA6aefznve8x5mzpzJpZdeyvHHH8/222/PnnvuyXrrrcf06dNfte9LLrmEa6+9lmnTpnHSSSdxxx13MHKkf10kSY1ljpSk9uA9hk12zz33cN99972yfMstt/DWt76VsWPHMnPmTIBX7pGA4rKa00477ZXlJ598ks0335xrrrmGBx98EOCVy2R22GEHvvvd75JZ3OB/8803A/DAAw+wxhprcNRRR7H77rsza9Ys5syZw7LLLssBBxzApz/9aW666SbWXntt5s6d+0rSe+mll7jjjjt4+eWXefjhh9lmm234+te/zrx583jmmWea2EuSpG5kjpSk9uHwVpM988wzHHnkkcybN4+RI0fy9re/nSlTpnDXXXdx2GGH8bWvfY1NN930lfZf+MIX+MQnPsH666/PiBEjOOGEE9hrr72YMmUKe+21Fy+//DIrr7wyV155JV/84hc55phjGD9+PJnJ2LFjufjiizn//PP56U9/ypJLLsmb3vQmvvSlL3HjjTfymc98hiWWWIIll1yS73//+yy11FL84he/4KijjmL+/PksWLCAY445hrXWWosDDjiA+fPnk5l88pOfZMUVV2xhL0qSOpE5UpLaR/SMpHWCiRMn5owZMxZZd9ddd7HOOuu0KKLOZt9KqkdD7584kZmZObFxO+xs5sehZ/9Kqkc75EgvJZUkSZKkLmdhKEmSJEldzsJQkiRJkrpc0yafiYipwK7Ao5m5frnufKBn3ugVgXmZOaHKex8CngYWAgu8j0SSJEmSmqeZs5KeCZwG/LhnRWbu0/M6Ir4FzO/n/dtk5mNNi06SpBZx8FSS1G6aVhhm5rURMbbatiieRPtB4D3NOr4kSW3sTBw8lSS1kVY9x/BdwD8z874+tidwRUQk8IPMnDJ0oTXeiBEjGDdu3CvL++67L8cdd1wLI5IktZKDpwXzoyS1j1YVhvsB5/azfcvMnBMRKwNXRsTdmXlttYYRcThwOMBqq6028JFPPLHuYBd3f695zWu45ZZb+m2zcOFCRowY8cryggULGDly4D+eWttJkoaNhgyemh/Nj5JUjyH/xoyIkcBewMZ9tcnMOeXvRyPiQmAToGphWCbEKVA8wLfhATfR2LFjOfTQQ7niiis44ogjmDx5MltssQXXX389u+++O3vvvTeHHnooc+fOZfTo0fzoRz9itdVW4+CDD+YNb3gDN998MxtttBG77747Rx99NAARwbXXXssKK6zQ4k8nSRqkhgyemh/Nj5JUj1YMpW0L3J2Zs6ttjIjlgCUy8+ny9fbAV4YywEZ7/vnnmTDh3/MHHH/88eyzT3EryTLLLMN1110HwOTJk5k3bx7XXHMNALvtthsf+tCHOOigg5g6dSpHHXUUv/rVrwC49957ueqqqxgxYgS77bYb3/ve99hyyy155plnWGaZZYb4E0qSGqHRg6ftzvwoSe2jmY+rOBeYBIyKiNnACZl5BrAvvUZCI2JV4PTM3Bl4I3BhcYsFI4FzMvOyZsU5FPq7VKYnAVZbnj59OhdccAEABx54IMcee+wr2z7wgQ+8cmnNlltuyac+9Sn2339/9tprL8aMGdPojyBJGhpdNXhqfpSk9tG0B9xn5n6ZuUpmLpmZY8qikMw8ODMn92o7pywKycwHMnOD8me9zDy5WTG2g+WWW67f5Uplsfyqdscddxynn346zz//PJttthl333134wOVJDVMOXg6HVg7ImZHxGHlpqqDpxFxabn4RuC6iLgV+DNwyXAfPO2L+VGShpZ3ZbexLbbYgvPOO48DDzyQs88+m6222qpqu7/85S+MGzeOcePGMX36dO6++27e8Y53DHG0kqRaZeZ+faw/uMq6OcArg6fABk0NbhgwP0pS41kYDoHe91DsuOOOnHLKKQO+79RTT+XQQw/lG9/4xis311fz7W9/m6uvvpoRI0aw7rrrstNOOzUsdkmSmsX8KEntIzKH1URl/Zo4cWLOmDFjkXV33XUX66yzTosi6mz2raR6xJdj4Ea1OpGZmTmxcTvsbObHoWf/SqpHO+TIpt1jKEmSJEkaHiwMJUmSJKnLWRhKkiRJUpfrisKwk+6jbBf2qSQNf36XN4f9Kmk46vjCcJllluHxxx/3S7qBMpPHH3+cZZZZptWhSJIGyfzYHOZIScNVxz+uYsyYMcyePZu5c+e2OpSOsswyyzBmzJhWhyFJGiTzY/OYIyUNRx1fGC655JKsvvrqrQ5DkqS2Yn6UJFXq+EtJJUmSJEn9szCUJEmSpC5nYShJkiRJXc7CUJIkSZK6nIWhJEmSJHU5C0NJkiRJ6nIWhpIkSZLU5SwMJUmSJKnLWRhKkiRJUpezMJQkSZKkLmdhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlLAwlSZIkqcs1rTCMiKkR8WhE3F6x7sSI+HtE3FL+7NzHe3eMiHsi4v6IOK5ZMUqSJEmSmnvG8Exgxyrr/zczJ5Q/l/beGBEjgO8BOwHrAvtFxLpNjFOSpCHl4Kkkqd00rTDMzGuBJwbx1k2A+zPzgcx8ETgP2KOhwUmS1Fpn4uCpJKmNtOIewyMiYlY5Wvr6KtvfDDxcsTy7XCdJUkdw8FSS1G6GujD8PvA2YALwCPCtKm2iyrrsa4cRcXhEzIiIGXPnzm1MlJIktUbDBk/Nj5KkegxpYZiZ/8zMhZn5MvBDipHP3mYDb6lYHgPM6WefUzJzYmZOHD16dGMDliRp6DR08NT8KEmqx5AWhhGxSsXinsDtVZrdCKwZEatHxFLAvsC0oYhPkqRWacbgqSRJtWrm4yrOBaYDa0fE7Ig4DPh6RNwWEbOAbYBPlm1XjYhLATJzAXAEcDlwF/CzzLyjWXFKktQOHDyVJLXSyGbtODP3q7L6jD7azgF2rli+FHjVbGySJHWCcvB0EjAqImYDJwCTImICxaWhDwH/UbZdFTg9M3fOzAUR0TN4OgKY6uCpJKkRmlYYSpKk6hw8lSS1m1Y8rkKSJEmS1EYsDCVJkiSpy1kYSpIkSVKXszCUJEmSpC5nYShJkiRJXc7CUJIkSZK6nIWhJEmSJHU5C0NJkiRJ6nIWhpIkSZLU5SwMJUmSJKnLWRhKkiRJUpezMJQkSZKkLmdhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlLAwlSZIkqctZGEqSJElSl7MwlCRJkqQuZ2EoSZIkSV3OwlCSJEmSupyFoSRJkiR1OQtDSZIkSepyFoaSJEmS1OUsDCVJkiSpyzWtMIyIqRHxaETcXrHuGxFxd0TMiogLI2LFPt77UETcFhG3RMSMZsUoSVIrmCMlSe2mmWcMzwR27LXuSmD9zBwP3Asc38/7t8nMCZk5sUnxSZLUKmdijpQktZGmFYaZeS3wRK91V2TmgnLxBmBMs44vSVK7MkdKktpNK+8xPBT4TR/bErgiImZGxOH97SQiDo+IGRExY+7cuQ0PUpKkFmhIjpQkqVYDFoYRsX6jDxoRnwcWAGf30WTLzNwI2An4RES8u699ZeaUzJyYmRNHjx7d6FAlSepTO+dIB04lSfWo5Yzh5Ij4c0R8vK8b4esREQcBuwL7Z2ZWa5OZc8rfjwIXApss7nElSWqCts2RDpxKkuoxYGGYmVsB+wNvAWZExDkRsd1gDhYROwKfBXbPzOf6aLNcRKzQ8xrYHri9WltJklrJHClJ6hQ13WOYmfcBX6BIWFsDp5ZTau/V13si4lxgOrB2RMyOiMOA04AVgCvLabYnl21XjYhLy7e+EbguIm4F/gxckpmXDfLzSZLUVOZISVInGDlQg4gYDxwC7EIxlfZumXlTRKxKkdQuqPa+zNyvyuoz+mg7B9i5fP0AsEFN0UuS1ELmSElSpxiwMKQYwfwh8LnMfL5nZWbOiYgvNC0ySZLanzlSktQRarmU9ILM/EllwouIowEy8ydNi0ySpPZnjpQkdYRaCsMPVVl3cIPjkCRpODJHSpI6Qp+XkkbEfsD/A1aPiGkVm1YAHm92YJIktStzpCSp0/R3j+EfgUeAUcC3KtY/DcxqZlCSJLU5c6QkqaP0WRhm5l+BvwKbD104kiS1P3OkJKnT9Hcp6XWZuVVEPA1k5SYgM/O1TY9OkqQ2ZI6UJHWa/s4YblX+XmHowpEkqf2ZIyVJnWbAWUkj4m0RsXT5elJEHBURKzY/NEmS2ps5UpLUKWp5XMUvgYUR8XbgDGB14JymRiVJ0vBgjpQkdYRaCsOXM3MBsCfw7cz8JLBKc8OSJGlYMEdKkjpCLYXhS+Xzmg4CLi7XLdm8kCRJGjbMkZKkjlBLYXgIxXTcJ2fmgxGxOvDT5oYlSdKwYI6UJHWE/h5wD0Bm3gkcVbH8IHBKM4OSJGk4MEdKkjrFgIVhRGwJnAi8tWzf84ymNZobmiRJ7c0cKUnqFAMWhhSzrH0SmAksbG44kiQNK+ZISVJHqKUwnJ+Zv2l6JJIkDT/mSElSR6ilMLw6Ir4BXAD8q2dlZt7UtKgkSRoehkeOnDMHTjyxMftq1H4kSW2llsJw0/L3xIp1Cbyn8eFIkjSsmCMlSR2hlllJtxmKQCRJGm7MkZKkTjHgcwwj4o0RcUZE/KZcXjciDmt+aJIktTdzpCSpU9TygPszgcuBVcvle4FjmhWQJEnDyJmYIyVJHaCWwnBUZv4MeBkgMxfglNySJIE5UpLUIWopDJ+NiJUobqYnIjYD5jc1KkmShgdzpCSpI9QyK+mngGnA2yLiemA0sHdTo5IkaXgwR0qSOkIts5LeFBFbA2sDAdyTmS81PTJJktqcOVKS1Cn6LAwjYq8+Nq0VEWTmBQPtPCKmArsCj2bm+uW6NwDnA2OBh4APZuaTVd57EPCFcvGrmXnWQMeTJGkoLG6OND9KktpNf/cY7lb+HAacAexf/pwOHFDj/s8Eduy17jjgt5m5JvDbcnkRZXI8geLBwZsAJ0TE62s8piRJzba4OfJMzI+SpDbSZ2GYmYdk5iEUN9Svm5nvz8z3A+vVuvPMvBZ4otfqPYCe0c2zgPdVeesOwJWZ+UQ5Wnolr06gkiS1xOLmSPOjJKnd1DIr6djMfKRi+Z/AWotxzDf27K/8vXKVNm8GHq5Ynl2ue5WIODwiZkTEjLlz5y5GWJIk1a2RObKh+VGSpHrUMivp7yPicuBcipHRfYGrmxpVcQN/b1mtYWZOAaYATJw4sWobSZKaZKhzZM35MSIOBw4HWO11r2tiSJKkTjDgGcPMPAKYDGwATACmZOaRi3HMf0bEKgDl70ertJkNvKVieQwwZzGOKUlSwzU4RzY0P2bmlMycmJkTRy+77CBDkiR1i1rOGJKZFwIXNuiY04CDgFPK3xdVaXM58LWKG+q3B45v0PElSWqYBuZI86MkqWVqucdw0CLiXGA6sHZEzI6IwygS3nYRcR+wXblMREyMiNMBMvMJ4CTgxvLnK+U6SZKGPfOjJKnd1HTGcLAyc78+Nr23StsZwIcrlqcCU5sUmiRJLWN+lCS1m5rOGEbEayJi7WYHI0nScGOOlCR1ggELw4jYDbgFuKxcnhAR05odmCRJ7c4cKUnqFLWcMTwR2ASYB5CZtwBjmxeSJEnDxomYIyVJHaCWwnBBZs5veiSSJA0/5khJUkeoZfKZ2yPi/wEjImJN4Cjgj80NS5KkYcEcKUnqCLWcMTwSWA/4F3AOMB84pplBSZI0TJgjJUkdoZYzhmtn5ueBzzc7GEmShhlzpCSpI9RyxvB/IuLuiDgpItZrekSSJA0f5khJUkcYsDDMzG2AScBcYEpE3BYRX2h2YJIktTtzpCSpU9T0gPvM/Edmngp8lOJ5TV9qalSSJA0T5khJUieo5QH360TEiRFxO3AaxWxrY5oemSRJbc4cKUnqFLVMPvMj4Fxg+8yc0+R4JEkaTsyRkqSOMGBhmJmbDUUgkiQNN+ZISVKn6LMwjIifZeYHI+I2ICs3AZmZ45senSRJbcgcKUnqNP2dMTy6/L3rUAQiSdIwYo6UJHWUPiefycxHypcfz8y/Vv4AHx+a8CRJaj/mSElSp6nlcRXbVVm3U6MDkSRpGDJHSpI6Qn/3GH6MYtRzjYiYVbFpBeD6ZgcmSVK7MkdKkjpNf/cYngP8Bvgv4LiK9U9n5hNNjUqSpPZmjpQkdZQ+C8PMnA/MB/YDiIiVgWWA5SNi+cz829CEKElSezFHSpI6zYD3GEbEbhFxH/AgcA3wEMUoqSRJXc0cKUnqFAM+4B74KrAZcFVmbhgR21COkEqSVKsTrm7cvr7cuF0tLnOkJKkj1DIr6UuZ+TiwREQskZlXAxOaHJckScOBOVKS1BFqOWM4LyKWB64Fzo6IR4EFzQ1LkqRhwRwpSeoItZwx3AN4HvgkcBnwF2C3ZgYlSdIwYY6UJHWEAc8YZuazFYtnNTEWSZKGFXOkJKlT9HnGMCKejoinKn6ervw92ANGxNoRcUvFz1MRcUyvNpMiYn5Fmy8N9niSJDWaOVKS1Gn6e47hCs04YGbeQ3ljfkSMAP4OXFil6R8yc9dmxCBJ0uIwR0qSOk0t9xgSEVtFxCHl61ERsXqDjv9e4C+Z+dcG7U+SpCFljpQkdYJaHnB/AvBZ4Phy1VLATxt0/H2Bc/vYtnlE3BoRv4mI9Rp0PEmSGsYcKUnqFLWcMdwT2B14FiAz5wCLfQlNRCxV7vfnVTbfBLw1MzcAvgv8qp/9HB4RMyJixty5cxc3LEmS6tG2OXKR/Pjcc4sbkiSpw9VSGL6YmQkkQEQs16Bj7wTclJn/7L0hM5/KzGfK15cCS0bEqGo7ycwpmTkxMyeOHj26QaFJklSTts2Ri+THZZdtUFiSpE5VS2H4s4j4AbBiRHwEuAo4vQHH3o8+LpGJiDdFRJSvNynjfLwBx5QkqZHMkZKkjlDLcwy/GRHbAU8BawNfyswrF+egEbEssB3wHxXrPloebzKwN/CxiFhA8eDgfcsRWUmS2oY5UpLUKQYsDAHKJHclFNNnR8T+mXn2YA+amc8BK/VaN7ni9WnAaYPdvyRJQ8UcKUnqBP094P61EXF8RJwWEdtH4QjgAeCDQxeiJEntxRwpSeo0/Z0x/AnwJDAd+DDwGYppuPfIzFuGIDZJktqVOVKS1FH6KwzXyMxxABFxOvAYsFpmPj0kkUmS1L7MkZKkjtLfrKQv9bzIzIXAgyY8SZIAc6QkqcP0d8Zwg4h4qnwdwGvK5QAyM1/b9OgkSWpP5khJUkfpszDMzBFDGYgkScOFOVKS1GlqecC9JEmSJKmDWRhKkiRJUpezMJQkSZKkLmdhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlLAwlSZIkqctZGEqSJElSl7MwlCRJkqQuZ2EoSZIkSV3OwlCSJEmSupyFoSRJkiR1OQtDSZIkSepyFoaSJEmS1OUsDCVJkiSpy1kYSpIkSVKXszCUJEmSpC5nYShJkiRJXa5lhWFEPBQRt0XELRExo8r2iIhTI+L+iJgVERu1Ik5JkiRJ6nQjW3z8bTLzsT627QSsWf5sCny//C1JUseKiIeAp4GFwILMnNhrewDfAXYGngMOzsybhjpOSVJnaXVh2J89gB9nZgI3RMSKEbFKZj7S6sAkSWoyB04lSUOqlfcYJnBFRMyMiMOrbH8z8HDF8uxynSRJ3eyVgdPMvAFYMSJWaXVQkqThrZWF4ZaZuRHFyOcnIuLdvbZHlfdk7xURcXhEzIiIGXPnzm1GnJIkDaWGDJwukh+fe65JoUqSOkXLCsPMnFP+fhS4ENikV5PZwFsqlscAc6rsZ0pmTszMiaNHj25WuJIkDZWGDJwukh+XXbYZcUqSOkhLCsOIWC4iVuh5DWwP3N6r2TTgQ+XspJsB872/UJLU6Ro1cCpJUj1adcbwjcB1EXEr8Gfgksy8LCI+GhEfLdtcCjwA3A/8EPh4a0KVJGloOHAqSWqVlsxKmpkPABtUWT+54nUCnxjKuCRJarE3AhcWT6RgJHBOz8ApvJInL6V4VMX9FI+rOKRFsUqSOkg7P65CkqSu4sCpJKlVWjkrqSRJkiSpDVgYSpIkSVKXszCUJEmSpC5nYShJkiRJXc7CUJIkSZK6nIWhJEmSJHU5C0NJkiRJ6nIWhpIkSZLU5SwMJUmSJKnLWRhKkiRJUpezMJQkSZKkLmdhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlLAwlSZIkqctZGEqSJElSl7MwlCRJkqQuZ2EoSZIkSV3OwlCSJEmSupyFoSRJkiR1OQtDSZIkSepyFoaSJEmS1OUsDCVJkiSpy1kYSpIkSVKXG/LCMCLeEhFXR8RdEXFHRBxdpc2kiJgfEbeUP18a6jglSRpq5khJUquMbMExFwD/mZk3RcQKwMyIuDIz7+zV7g+ZuWsL4pMkqVXMkZKklhjyM4aZ+Uhm3lS+fhq4C3jzUMchSVK7MUdKklqlpfcYRsRYYEPgT1U2bx4Rt0bEbyJivX72cXhEzIiIGXPnzm1SpJIkDa3FzZGL5MfnnmtipJKkTtCywjAilgd+CRyTmU/12nwT8NbM3AD4LvCrvvaTmVMyc2JmThw9enTzApYkaYg0Ikcukh+XXba5AUuShr2WFIYRsSRFwjs7My/ovT0zn8rMZ8rXlwJLRsSoIQ5TkqQhZ46UJLVCK2YlDeAM4K7M/J8+2rypbEdEbEIR5+NDF6UkSUPPHClJapVWzEq6JXAgcFtE3FKu+xywGkBmTgb2Bj4WEQuA54F9MzNbEKskSUPJHClJaokhLwwz8zogBmhzGnDa0EQkSVJ7MEdKklqlpbOSSpIkSZJarxWXkkqShosTT2x1BJIkaQh4xlCSJEmSupxnDCVJUu0aeRbZM9KS1DY8YyhJkiRJXc7CUJIkSZK6nIWhJEmSJHU57zGUJEmt4f2KktQ2PGMoSZIkSV3OM4bScOeIuyRJkhaTZwwlSZIkqct5xlCSOo1nfiVJUp08YyhJkiRJXc4zhpLUDjzLJ0mSWsjCUNK/OZGNJPldKKkrWRhK0mD5Hz6pffjvUZIWi/cYSpIkSVKX84yhpOZw9F6SJGnYsDCUJElqlnYdJGvX+yjbtb+kLuClpJIkSZLU5SwMJUmSJKnLeSmpJElSt/GSTUm9WBhKrWBCliRJUhuxMJQk9enE33+51SFI6iZOZKNhJL4crQ6hobzHUJIkSZK6nIWhJEmSJHW5llxKGhE7At8BRgCnZ+YpvbYvDfwY2Bh4HNgnMx8a6jjVAbyMRF3Iyz+HN3OkJDVPp13+2UhDXhhGxAjge8B2wGzgxoiYlpl3VjQ7DHgyM98eEfsC/w3sM9SxSlJ/LMDUaOZISVqUhdzQacUZw02A+zPzAYCIOA/YA6hMensAJ5avfwGcFhGRmTmUgapFPMtXF4sTqcB5G8QAABbZSURBVKOYI6UGsaCQ6tOKwvDNwMMVy7OBTftqk5kLImI+sBLwWO+dRcThwOHl4r8i4vaGR9y5RlGlT9Un+6s+9ld97K/6rN3qAJqkYTnyVfnxy182P9bOf4/1sb/qY3/Vx/6q36ByZCsKw2rDN71HOWtpU6zMnAJMAYiIGZk5cfHC6x72V33sr/rYX/Wxv+oTETNaHUOTNCxHmh8Hz/6qj/1VH/urPvZX/QabI1sxK+ls4C0Vy2OAOX21iYiRwOuAJ4YkOkmSWsccKUlqiVYUhjcCa0bE6hGxFLAvMK1Xm2nAQeXrvYHfee+EJKkLmCMlSS0x5JeSlvdDHAFcTjEV99TMvCMivgLMyMxpwBnATyLifopR0H1r3P2UpgTdueyv+thf9bG/6mN/1acj+6uJObIj+6uJ7K/62F/1sb/qY3/Vb1B9Fg4ySpIkSVJ3a8WlpJIkSZKkNmJhKEmSJEldbtgVhhGxY0TcExH3R8RxVbYvHRHnl9v/FBFjhz7K9lFDf30qIu6MiFkR8duIeGsr4mwnA/VZRbu9IyIjoqunUK6lvyLig+Xfszsi4pyhjrGd1PBvcrWIuDoibi7/Xe7cijjbQURMjYhH+3o+bRROLftyVkRsNNQxthtzZH3MkfUxP9bH/Fgf82N9mpIjM3PY/FDciP8XYA1gKeBWYN1ebT4OTC5f7wuc3+q427y/tgGWLV9/rJv7q9Y+K9utAFwL3ABMbHXc7dxfwJrAzcDry+WVWx13m/fXFOBj5et1gYdaHXcL++vdwEbA7X1s3xn4DcVz/TYD/tTqmFvcX+bIxveXObKO/irbmR9r7C/zY939ZX5ctD8aniOH2xnDTYD7M/OBzHwROA/Yo1ebPYCzyte/AN4bEdUeBtwNBuyvzLw6M58rF2+geGZWN6vl7xjAScDXgReGMrg2VEt/fQT4XmY+CZCZjw5xjO2klv5K4LXl69fx6mfYdY3MvJb+n8+3B/DjLNwArBgRqwxNdG3JHFkfc2R9zI/1MT/Wx/xYp2bkyOFWGL4ZeLhieXa5rmqbzFwAzAdWGpLo2k8t/VXpMIqRhW42YJ9FxIbAWzLz4qEMrE3V8ndsLWCtiLg+Im6IiB2HLLr2U0t/nQgcEBGzgUuBI4cmtGGp3u+4TmeOrI85sj7mx/qYH+tjfmy8unPkkD/HcDFVG9Xs/byNWtp0i5r7IiIOACYCWzc1ovbXb59FxBLA/wIHD1VAba6Wv2MjKS6XmUQx2v6HiFg/M+c1ObZ2VEt/7QecmZnfiojNKZ5Xt35mvtz88IYdv+8XZY6sjzmyPubH+pgf62N+bLy6v++H2xnD2cBbKpbH8OrTyK+0iYiRFKea+zvN2slq6S8iYlvg88DumfmvIYqtXQ3UZysA6wO/j4iHKK7ZntbFN9jX+m/yosx8KTMfBO6hSITdqJb+Ogz4GUBmTgeWAUYNSXTDT03fcV3EHFkfc2R9zI/1MT/Wx/zYeHXnyOFWGN4IrBkRq0fEUhQ3zk/r1WYacFD5em/gd1negdmFBuyv8rKPH1AkvG6+tr1Hv32WmfMzc1Rmjs3MsRT3nOyemTNaE27L1fJv8lcUEzgQEaMoLp15YEijbB+19NffgPcCRMQ6FIlv7pBGOXxMAz5Uzry2GTA/Mx9pdVAtZI6sjzmyPubH+pgf62N+bLy6c+SwupQ0MxdExBHA5RSzF03NzDsi4ivAjMycBpxBcWr5fopR0H1bF3Fr1dhf3wCWB35ezj/wt8zcvWVBt1iNfaZSjf11ObB9RNwJLAQ+k5mPty7q1qmxv/4T+GFEfJLiko+Du/U/7hFxLsUlVqPKe0pOAJYEyMzJFPeY7AzcDzwHHNKaSNuDObI+5sj6mB/rY36sj/mxfs3IkdHF/SlJkiRJYvhdSipJkiRJajALQ0mSJEnqchaGkiRJktTlLAwlSZIkqctZGEqSJElSl7MwlJogIt4UEedFxF8i4s6IuDQi1hrEft4VEXdExC0R8eaI+EUf7X7fxQ8RliQNI+ZIqT1ZGEoNFsXDri4Efp+Zb8vMdYHPAW8cxO72B76ZmRMy8++ZuXcjY5UkaSiZI6X2ZWEoNd42wEvlw0UByMxbgOsi4hsRcXtE3BYR+wBExKRyNPMXEXF3RJwdhQ8DHwS+VK4bGxG3l+95TTnaOisizgde03OsiNg+IqZHxE0R8fOIWL5c/1BEfLlcf1tEvKNcv3xE/KhcNysi3t/ffiRJWgzmSKlNWRhKjbc+MLPK+r2ACcAGwLbANyJilXLbhsAxwLrAGsCWmXk6MA34TGbu32tfHwOey8zxwMnAxgARMQr4ArBtZm4EzAA+VfG+x8r13wc+Xa77IjA/M8eV+/tdDfuRJGkwzJFSmxrZ6gCkLrIVcG5mLgT+GRHXAO8EngL+nJmzASLiFmAscF0/+3o3cCpAZs6KiFnl+s0oEuf1xdU6LAVMr3jfBeXvmRRJGIoEvG9Pg8x8MiJ2HWA/kiQ1kjlSajELQ6nx7gCq3ecQ/bznXxWvF1Lbv83s4xhXZuZ+Axyn8hhRZV8D7UeSpMEwR0ptyktJpcb7HbB0RHykZ0VEvBN4EtgnIkZExGiKEc0/D/IY11LcdE9ErA+ML9ffAGwZEW8vty1bw0xvVwBHVMT6+kHuR5KkgZgjpTZlYSg1WGYmsCewXRRTcd8BnAicA8wCbqVIjMdm5j8GeZjvA8uXl8ccS5k8M3MucDBwbrntBuAdA+zrq8Dryxv+bwW2GeR+JEnqlzlSal9R/PuUJEmSJHUrzxhKkiRJUpezMJQkSZKkLmdhKEmSJEldzsJQkiRJkrqchaEkSZIkdTkLQ0mSJEnqchaGkiRJktTlLAw1rEXEmRFxcavjaLSIGBsRGRETWx1LozXiz6yW/undZqBlSVLzRMRHI+KxVsfRDBHxj4g4YjH3MWD/9G4z0LJULwtDNUz5H/6MiNOrbPt6ua3RRdzRwAEN3mfTRcSEiDi/TCYvRMT9Zf+Na3VsABExqfzz6vmZGxG/iYgNWh1bHR4GVgFuqWV7xWceNUTxSVJden0vV/s5s4WxfTAifh8R8yPimYiYFRFfbpfv1Ig4paKfFkbE3yJickS8odWx1eEsYN1at5efeUbTo1LHsDBUoz0M7BMRy/WsiIiRwIHA3xp9sMycn5nzGr3fZoqIXYE/ActT9Ms6wL7AI8ApLQytmvUoiqddgNcDl0XE66o1jIilhjKwgWTmwsz8R2YuGMx2SWpDq1T8fKTKuqOrvSkilmxmUBHxLeAc4M/AThS541MU+e2wZh67TrdS9NNbgaOADwBn9NW42f1Wr8x8PjMfHex2aSAWhmq0WcB9wAcr1u0CvAD8vrJhRLwzIq6IiMci4qmIuC4iNq/YvnVEvBQRkyrWfbRsu0a5vMhlieVo5fcj4lsR8UR5puvoiFg6Ir4XEfPKUcIDK95T9ZLCct3evdrsGxHXRMTzEXFzRIyPiPUj4o8R8Wz5GVbvq3MiYlngR8DlmblLZl6ZmQ9m5ozMPB7Yv4/3jYiIMyLiwfLY90XEsRGxREWbcRHx27J/no6IWyNim3LbkhFxakTMiYh/RcTDEVFLEfpoWTz9GfhP4E3AZuU+H4qIEyNiakTMA86uiOOqMs4nyj+jVxWTEfGFiPhnObL8o4h4TcW2HSPiDxHxZLmPyyNinSrxrVX2+QsRcXdEbF+xj34vFa3cHhFjgavLTXN7Rt4j4kMR8XhELN3rvWdHxLQa+k+SGqb8Pv5HZv4DmNd7XWbOj4h3lN9hHyjz1QvAQVHlMsPyuzYjYvmKde8uv1efL3PFdyu39xYR76YoAo/KzGMz84+Z+dfMvCozPwj8oI/3vSMifl2RB2ZUfoeXbfaJiNvLWB6PiKsjYqVy2+oRcXGZJ56NiDsjYq8BunBB2U+zM/NXwP8Bu5Q5smq/VcRxZ0S8GBF/jYhjq+z7dRFxXhnLnIg4qtdn+Wz5WZ4t+/X7EfHaKv3y/jLHvxARV0bEahXb+r1UtHJ7RHwU+Cywcfz7TOm+EXFORPyi1/tGRnEF08cH6D91OAtDNcMZwKEVy4dSFEPZq90KwE+AdwGbUFzSd2mUl51k5jXAN4CfRMQbIuIdwLeAIzPzgX6Ovz/wNLApxRm4bwO/Au4FJlJcanF6RKw6iM/2ZeC/gQ0pkvI5wHeBz5efYRng1H7evwMwij7ODPZz9nMJ4O8UBfc65fE+BxxS0eYcirOOm5TxnUhRkEMxMronxZnJNYF9gHv6ibOa58vflSOonwLupujXz0VR+F4GPFPGsSewBTC11762BjYA3gu8H9ieol97LEfx57YJMAmYD/w6Xn1W8usU/T0BuBK4KCLeXOfnguJM9/vL1z1nSY8Gfk7R93v0NCyL3D3pZ5RZktrAKcD/UuSMS2t5Q0RsDPwG+BkwjiLnbA5M7udt+wNP0kcB2E9eWx6YRpEHNgQuofie7xn4fSvFgOPk8jNMAs6reP8UIIB3l7F+GnhqgI/Y2/MU3/EjKtYt0m8RsQVwbhnL+sAJwIkR8ZFe+zoWuKn8LF8DvhURO1dsXwAcQZFjPkSRB7/Vax8rUBRzBwJbUvTRLxics4DT+PdZ0lUo/i/0Q2C3WPQS312A11H8P0LdLDP98achP8CZwMUUlxw+T1GAvAn4F7Baz/Z+3h8Uhc0BFeuWBG4ELqD4wj2/2jErln8PTO+1z7nAtF77fBHYu1weS1G0Tuy176zS5j8qtu9arturYt3BwDP9fMZjy/e8foC+rBpTrzanAFdVLD8FHNRH21OB3wJR45/lpPL4o8rllYCLymOsXK57CPh1r/d9hKKIW6HKvt5e8Wc2D1i+os0B5d+T5fqIZzlgIbBVr/75fEWbJSiK/69W68Malhf5zBX7PQ24rGL5Y8A/gJGt/jfnjz/+dO8PsDeQVda/o/wu+0Sv9R8FHuu1bsey7fLl8s+A7/Vqs1nZ5rV9xPE74E81xPuq41dpcwvw6fL1FsDLwJv6aHsv8Nk6+usUYEbF8nplHrtmgH77JXBplX3dX7H8jyr58KdU5Ogq8bwPeLpX/ySwccW6Nct1W1W0eazXe/pbXuQzl+uC4squYyrWXQT8tNV/p/1p/Y9nDNVwmfkkcCHFmcKDgN9n5qvuL4yIlSPiBxFxb0TMpzjLtzJFEdmzr5eA/0dRhK0M/EcNIcyqeH8CjwK39drnk+X+6jWr4vU/y9+39Vq3XHnmrJoYxDGLNxaXiMyI4vLYZ4BPUtFXwP9QnAn9XUR8vjzD2uNMirNq90ZxSe0uUXEZaj8eKo/1GMXo6Qdy0fsXet/Uvg4wKzOfrlj3R4rkXnnD/KzMfKZieTqwFPC28rO+rbzc5S8R8RRFvy7R6/P2vA+AzHyZ4t7N/m7MH4wfAttFxJhy+VDgrPTeREntbTCTjmwMfLi8tPOZ8vv/t+W2t/XxnkHltYh4bUT8T0TcFcVtHs9QnJHr+Z6/EfgDcE9E/DwiDu+5jLT0beCrEXF9RHwlIibUcNgNy8/1PEXuvo/yctEK1fLa9b3WXQesERHLVKyb3qvNdBadCGb7Mj//PSKepjg7t3wsOvnNixSD4ABk5n0U+bdhea38f9HplFd2RcQbgZ3xKhjhpaRqnqkUl0ocyqsvI+xxFvBOigJnC4rCZTZFgVBpM4q/qysCo2s49ku9lrOPdT1//18uf7+S3KLvG84r95P9rOvr39a95e9q98v1KSL2oUiCZ1JcjjqB4t6IV/oqM0+kSB6/oujPWRFxaLntJoozZJ8rYzsLuLKG4nAbiks+X5eZa2Xm5b22P9s7VF59yfArIQ5wrEq/pviz/g+KS4I3pLgMZ8gnuMnMWykS9cERsT7FZbN9/Z2WpHbR+/v5ZV5dxPXOdUsA36PIMT0/G1Ccubqzj+PcC6wZESP62N6X7wC7UeSld5XHuoXye74cxN2Gomi5k+JqjfuivN88M/+Polj9CUXu+3NEHDfAMe8sj7MO8JrM3C4zH+rVpta8ln2sf5WIWJMir91McdvCxhRn92DRvFZPnlwcZwLrRHEP/ocoJgf8/RAdW23MwlDN8luKka9RFIVKNVsB383MSzLzDoozhqtUNohiUpDTgE9Q3EN2dhSznDbS3PJ35bFrGXkcjCsoRv+qJq+IWLGP921FcanOaZl5U2beT5XR28y8LzNPzcxdKEb/Plyx7enM/HlmfozifoL3AG8fIN4HM/MvmVnrfRt3AhtExAoV67ag+K65q2LduKiYuZai+H8R+Es5IrwO8LUsJi+4i+K+i2p/7pv1vIiIoLgn8a4q7WrxYvm72n9ufkhxmfCHgeszs977MyWp1eYCK/Y6y9U7190ErJeZ91f5+Vcf+z2H4haSqlf0DJDXpmbmhZl5G8WtJGtUNsjMlzPz+sw8gaKYepJiJtGe7X/LzMmZuTdwMnB4H8fq8a/yszzUz+fp7c4y1t6xP9BrH5v1arMZ/85HmxTh5n9m5g2ZeS8whldbmmIgFCiunqH4f9Ti5LVX5bTM/CdFoXooxVwFPyrPJKrLWRiqKcovmPHA6v18+d4LHBAR60bEOyluKu/5zznl6ONPKa7//wHFf8rHUNz43chYnwduAD4bEeuVN5p/s5HHqDjWsxSfY8eIuCQitotidsyNIuIkypk9q7gX2CgidoqINSPiixQ3rgMQEa8pLxGdVO5vU4rEdWe5/VMRsV9ErBMRb6e4PPcpijO0jXQ2xWjrj6OYnfTdFBMSXFAWsz1GAlPL/t6O4j6IH5b98yRF8fyRiHh7RGxNMflAtUs3PxYRe0fE2hRnVN8KfH+Qsf+VYrR2l4gYHYvOwncuxf2yH8PLbSQNT3+kyLH/VX637sO/H3nR42vApChmsZ5Q5pvdI+J7fe00i4niTgW+ExH/HRGbR8RbI+I9EXEefd8Cci/w/ojYIIpn5J5LxQBgRLwrIo6PYubo1Sgm/VqFf+e108rLM1ePiI2A7ej7rObi+CZFzv5c2R8HUUzo9vVe7baOiE+XbT5OMdnb/5bb7gOWjogjyngPBKrNAPoi8H8RsWn5mX5McY/gHwYZ+0PA26KYQX1ULDqB2w8pHiWyNsUZRMnCUM1TnqHq70zToRQzbs2kKAqnUnyJ9fgcxRmtw8r9PU5xL8BxEdF79G5x9cyieiNFIfOFBu//FZl5EcUsb89RFL73UMx++RaKyWmq+QHFpADnlDGOZdHZzBZSjNieVe7vQor7Gz5Vbn8a+AzFM6Zuohgl3ikzn2vQxwKg3N8OwGvLY11UxnFor6bXAHdQPCLiQorJC44t9/Eyxayp44HbKS5r+iLF5DS9HUfxGW+lmERhz8wcVLGbmX+nGHQ4meKextMqtj1N0f8vlr8laVgpzxJ9iOLyzdvK11/q1WYmxaDjOhT30d0MfJVicpX+9n10ub/NgcspCrTvUOSjH/bxtiMpZrCeTjFx3VUUeaPHPIpJwS6lKCL/i2LCsZ5ZOpekGAi8i2I27L/ShGcmZuZ0YD+KSdLuAE4CTszM3p/r6xRnCW+h6NdjM/Pich9/psjBnyv3cQDF7KO9PU2R28+l6JcXqDhDOgjnU+TXaynOGFc+zuNyijkYLh9s3lTnCc8cS9LAIuI3wOzM7D3CLknSsFJeFTMHOCQzf9nqeNQeGn2vliR1lHLGuG0pnrW4QYvDkSRp0MrbdN5IcZXO4/Q9D4S6kIWhJPXvJuANwOcy8/ZWByNJ0mJYk+Ly279RPPt4YYvjURvxUlJJkiRJ6nJOPiNJkiRJXc7CUJIkSZK6nIWhJEmSJHU5C0NJkiRJ6nIWhpIkSZLU5SwMJUmSJKnL/X9J3MV+MT4AqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15,5))\n",
    "plt.subplot(121)\n",
    "plt.title('Maximum Class Probability', y=-0.2, fontsize=14)\n",
    "plt.hist(confidence_normal[np.where(accurate_normal==1)], \n",
    "         bins=np.linspace(0, 1,num=21), density=True, color='green',label='Successes')\n",
    "plt.hist(confidence_normal[np.where(accurate_normal==0)], \n",
    "         bins=np.linspace(0, 1,num=21), density=True, alpha=0.5, color='red',label='Errors')\n",
    "plt.xlabel('Confidence')\n",
    "plt.ylabel('Relative density')\n",
    "plt.xlim(left=0, right=1)\n",
    "plt.legend()\n",
    "plt.subplot(122)\n",
    "plt.title('True Class Probability', y=-0.2, fontsize=14)\n",
    "plt.hist(confidence_gt[np.where(accurate_gt==1)], \n",
    "         bins=np.linspace(0, 1,num=21), density=True, color='green',label='Successes')\n",
    "plt.hist(confidence_gt[np.where(accurate_gt==0)], \n",
    "         bins=np.linspace(0, 1,num=21), density=True, alpha=0.5, color='red',label='Errors')\n",
    "plt.xlabel('Confidence')\n",
    "plt.ylabel('Relative density')\n",
    "plt.xlim(left=0, right=1)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
